﻿#include "tokenizer.h"
#include "ds/list.h"
#include "global.h"
#include "mem.h"

//bug #include "my_my_snprintf.h"

#include "snprintf.h"

#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

//必须windows换行模式,否则中文各种报错

static Token* token(TokenType type, char* literal, int line, int column, char* lexeme)
{
    size_t length = 0;
    Token* tokn = (Token*)alloc(sizeof(Token));
    tokn->type = type;
    if (literal == NULL)
    {
        tokn->literal = NULL;
    }
    else
    {
        length = strlen(literal) + 1;
        tokn->literal = (char*)alloc(length);
        #ifdef _WIN32
        strncpy_s(tokn->literal,length, literal, length); //目标的内存在哪里分配的呢,上面行呢
        #else
        strncpy(tokn->literal, literal, length);   
        #endif
    }
    if (lexeme == NULL)
    {
        tokn->lexeme = NULL;
    }
    else
    {
        length = strlen(lexeme) + 1;
        tokn->lexeme = (char*)alloc(length);
        #ifdef _WIN32
        strncpy_s(tokn->lexeme,length, lexeme, length);//两个一样长 length
        #else
        strncpy(tokn->literal, literal, length);   
        #endif
    }
    tokn->line = line;
    tokn->column = column;
    return tokn;
}

static Token* token_simple(TokenType type, int line, int column, char* lexeme)
{
    return token(type, NULL, line, column, lexeme);
}

static void token_destroy(Token* tokn)
{
    fr(tokn->literal);
    fr(tokn->lexeme);
    fr(tokn);
}

static void toknzr_error(int line, int column, char c)
{
    char buf[LINEBUFSIZE];
    memset(buf, 0, LINEBUFSIZE);
    my_snprintf(buf, LINEBUFSIZE, "Syntax Error at (%d, %d): %c is unexpected", line, column + 1, c);
    puts(buf);
}

static int match_next(const char* code, char next, size_t length, size_t* current)
{
    if (current == NULL || (*current) > length)
    {
        return 0;
    }

    if (code[(*current) + 1] != next)
    {
        return 0;
    }

    (*current)++;
    return 1;
}

//steven+
//格式 i1_1234 这种格式,跳过前面3字节
//扫描合法的整数字符串,然后付出出来返回
static char* read_int(const unsigned char* code, size_t codeLength, size_t* current ,int* token_int)
{
    char* literal = NULL;
    int start = *current + 3 , length = 0;
    char token_inttype= code[*current] ;//是i还是u
    char token_intn= code[*current+1];//1,2,4,8是哪个
    
   //printf("DEBUGintn=%c token_inttype %c \n",token_intn,token_inttype);//bug
    
    (*current) =(*current) + 3;
    
    //第一个字符是不是负数
    if((*current)=='-')
       (*current)++; 
    do
    {
        (*current)++;
    }
    while (!IS_AT_END(*current, codeLength) && isdigit(code[*current]));

    length = *current - start + 1;
    (*current)--;
    literal = (char*)alloc(length);
    memcpy(literal, &(code[start]), length);
    literal[length - 1] = '\0';

    //printf("DEBUGG %d   %s \n",length ,literal);

    //判断是什么整数
    if(token_inttype=='u' && token_intn=='1')
    {
        *token_int=TOKEN_U1;
        //printf("DEBUGH TOKEN_U1 \n");
    }
    if(token_inttype=='u' && token_intn=='2')
    {
        *token_int=TOKEN_U2;
        //printf("DEBUGH TOKEN_U2 \n");
        
    }
    if(token_inttype=='u' && token_intn=='4')
    {
        *token_int=TOKEN_U4;
        //printf("DEBUGH  TOKEN_U4\n");
        
    }
    if(token_inttype=='u' && token_intn=='8')
    {
        *token_int=TOKEN_U8;
        //printf("DEBUGH  TOKEN_U8 \n");
        
    }
    if(token_inttype=='i' && token_intn=='1')
    {
        *token_int=TOKEN_I1;
        //printf("DEBUGH TOKEN_I1 \n");
        
    }
    if(token_inttype=='i' && token_intn=='2')
    {
        *token_int=TOKEN_I2;
       //printf("DEBUGH TOKEN_I2 \n ");
        
    }
    if(token_inttype=='i' && token_intn=='4')
    {
        *token_int=TOKEN_I4;
        //printf("DEBUGH TOKEN_I4\n");
        
    }
    if(token_inttype=='i' && token_intn=='8')
    {
        *token_int=TOKEN_I8;
        //printf("DEBUGH  TOKEN_I8\n");
        
    }


    return literal;
}

static char* read_number(const unsigned char* code, size_t codeLength, size_t* current,int* token_int_float)
{
    char* literal = NULL;
    int start = *current, length = 0;
    int float_flag;
    
    if((*current)=='-')
       (*current)++; 
   
    do
    {
        (*current)++;
    }
    while (!IS_AT_END(*current, codeLength) && isdigit(code[*current]));
    float_flag=0;
    if (code[*current] == '.' && isdigit(code[(*current) + 1]))
    {
        //steven+自动判断有小数点,就是浮点数,没有就是整数
        if(code[*current] == '.')float_flag=1;
        do
        {
            (*current)++;
        }
        while (!IS_AT_END(*current, codeLength) && isdigit(code[*current]));
    }
    length = *current - start + 1;
    (*current)--;
    literal = (char*)alloc(length);
    memcpy(literal, &(code[start]), length);
    literal[length - 1] = '\0';
    
    //printf("DEBUG %s \n",literal);
    
    if(float_flag==1)
        *token_int_float=TOKEN_NUMBER;
    else
        *token_int_float=TOKEN_I8;
    
    return literal;
}


static char* read_other(const unsigned char* code, size_t codeLength, size_t* current)
{
    char* literal = NULL;
    int start = *current, length = 0;
    do
    {
        (*current)++;
    }
    while (!IS_AT_END(*current, codeLength) && IS_ALPHA_NUMERIC(code[*current]));
    length = *current - start + 1;
    (*current)--;
    literal = (char*)alloc(length);
    memcpy(literal, &(code[start]), length);
    literal[length - 1] = '\0';
    return literal;
}

Tokenization toknzr(const unsigned char* code, int verbose)
{
    char* literal = NULL;
    Token* tokn = NULL;
    TokenType type = TOKEN_ENDOFFILE;
    size_t length = strlen(code);
    size_t current = 0, start = 0, line = 1;
    int token_int_float;
    
    Tokenization toknz;
    toknz.values = list();
    toknz.lines = 0;
    
    while (!IS_AT_END(current, length))
    {
        unsigned char c = code[current];
        switch (c)
        {
        case '(':
            tokn = token_simple(TOKEN_LEFT_PAREN, line, current, (char*)"(");
            break;
        case ')':
            tokn = token_simple(TOKEN_RIGHT_PAREN, line, current, (char*)")");
            break;
        case '{':
            tokn = token_simple(TOKEN_LEFT_BRACE, line, current, (char*)"{");
            break;
        case '}':
            tokn = token_simple(TOKEN_RIGHT_BRACE, line, current, (char*)"}");
            break;
        case ',':
            tokn = token_simple(TOKEN_COMMA, line, current, (char*)",");
            break;
        case '.':
            tokn = token_simple(TOKEN_DOT, line, current, (char*)".");
            break;
        case '-':
            tokn = token_simple(TOKEN_MINUS, line, current, (char*)"-");
            break;
        case '+':
            tokn = token_simple(TOKEN_PLUS, line, current, (char*)"+");
            break;
        case ';':
            tokn = token_simple(TOKEN_SEMICOLON, line, current, (char*)";");
            break;
        case '*':
            tokn = token_simple(TOKEN_STAR, line, current, (char*)"*");
            break;
        case '!':
            type = match_next(code, '=', length, &current) ? TOKEN_BANG_EQUAL : TOKEN_BANG;
            tokn = token_simple(type, line, current, (char*)"!");
            break;
        case '=':
            type = match_next(code, '=', length, &current) ? TOKEN_EQUAL_EQUAL : TOKEN_EQUAL;
            tokn = token_simple(type, line, current, (char*)"=");
            break;
        case '>':
            type = match_next(code, '=', length, &current) ? TOKEN_GREATER_EQUAL : TOKEN_GREATER;
            tokn = token_simple(type, line, current, (char*)">");
            break;
        case '<':
            type = match_next(code, '=', length, &current) ? TOKEN_LESS_EQUAL : TOKEN_LESS;
            tokn = token_simple(type, line, current, (char*)"<");
            break;
        case '/':
            type = match_next(code, '/', length, &current) ? TOKEN_ENDOFFILE : TOKEN_SLASH;
            if (type == TOKEN_ENDOFFILE)
            {
                do
                {
                    current++;
                }
                while (current != length && code[current] != '\n');
            }
            else
            {
                tokn = token_simple(TOKEN_SLASH, line, current, (char*)"/");
            }
            break;
        case '"':
            start = current;
            do
            {
                if (code[current] == '\n')
                {
                    line++;
                }
                current++;
            }
            while (code[current] != '"' && !IS_AT_END(current, length));
            if (IS_AT_END(current, length))
            {
                if (verbose)
                {
                    toknzr_error(line, current, code[current]);
                }
                else
                {
                    tokn = token_simple(TOKEN_ERROR, line, current, "Unterminated string.");
                }
            }
            else
            {
                literal = (char*)alloc(current - start);
                memcpy(literal, &(code[start + 1]), current - start);
                literal[current - start - 1] = 0;
                if (literal != NULL)
                {
                    tokn = token(TOKEN_STRING, literal, line, current, literal);
                }
            }
            break;
        case ' ':
        case '\r':
        case '\t':
            break;
        case '\n':
            line++;
            break;
        default:
            //steven+考虑这里判断是否是整数,当是整数,
            //有可能用 i1,i2,i4,i8,u1,u2,u4,u8,作为前缀表达
            //如果没有这些前缀就表达默认的双精度浮点数
            //比如 i1_123 表示8bit的(1byte)的数据
            //printf("DEBUG %c \n",c);
            //printf("DEBUGREADINT c=%c   %c %c %d %d\n  ",c,code[current+1],
            //code[current+2],
            //current,
            //IS_AT_END(current+1, length) );
            //
            //f((c=='i' || c=='u'))
            //   printf(" %d %d %d %d",!IS_AT_END(current+1, length), 
            //    (code[current+1]=='1' ||code[current+1]=='2' ||code[current+1]=='4' ||code[current+1]=='8'   ),
            //    (!IS_AT_END(current+2, length) &&
            //           (code[current+2]=='_' )
            //          ),(!IS_AT_END(current+3, length) &&
            //           (isdigit(code[current+3])||code[current+3]=='-')  
            //          ));
            //    
            
            if((c=='i' || c=='u')
                    && (!IS_AT_END(current+1, length) &&
                        (code[current+1]=='1' ||code[current+1]=='2' ||code[current+1]=='4' ||code[current+1]=='8'   )
                       )
                    && (!IS_AT_END(current+2, length) &&
                        (code[current+2]=='_' )
                       )
                    && (!IS_AT_END(current+3, length) &&
                        (isdigit(code[current+3])||code[current+3]=='-') //要支持负数
                       )
              )
            {
                int token_int;
                //printf("goto read int ");
                literal = read_int(code, length, &current,&token_int);//读取后面的数字,不要读前缀, 返回token类型TOK_I1等
                tokn = token(token_int, literal, line, current, literal);
               //printf("DEBUG token_int = %d literal=%s \n",token_int, literal);
            }
            else //bug steven
            if (isdigit(c))//不能在这里 ||c=='-')//支持负数
            {
                literal = read_number(code, length, &current,&token_int_float);
                //tokn = token(TOKEN_NUMBER, literal, line, current, literal);
                tokn = token(token_int_float, literal, line, current, literal);
                //printf("read number = %s   %d ",literal ,token_int_float);
            }
            else if (isalpha(c))
            {
                literal = read_other(code, length, &current);
                if (strcmp(literal, AND_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_AND, line, current, (char*)AND_KEY);
                }
                else if (strcmp(literal, CLASS_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_CLASS, line, current, (char*)CLASS_KEY);
                }
                else if (strcmp(literal, ELSE_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_ELSE, line, current, (char*)ELSE_KEY);
                }
                else if (strcmp(literal, FALSE_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_FALSE, line, current, (char*)FALSE_KEY);
                }
                else if (strcmp(literal, FUN_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_FUN, line, current, (char*)FUN_KEY);
                }
                else if (strcmp(literal, FOR_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_FOR, line, current, (char*)FOR_KEY);
                }
                else if (strcmp(literal, IF_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_IF, line, current, (char*)IF_KEY);
                }
                else if (strcmp(literal, NIL_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_NIL, line, current, (char*)NIL_KEY);
                }
                else if (strcmp(literal, OR_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_OR, line, current, (char*)OR_KEY);
                }
                else if (strcmp(literal, PRINT_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_PRINT, line, current, (char*)PRINT_KEY);
                }
                else if (strcmp(literal, RETURN_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_RETURN, line, current, (char*)RETURN_KEY);
                }
                else if (strcmp(literal, SUPER_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_SUPER, line, current, (char*)SUPER_KEY);
                }
                else if (strcmp(literal, THIS_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_THIS, line, current, (char*)THIS_KEY);
                }
                else if (strcmp(literal, TRUE_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_TRUE, line, current, (char*)TRUE_KEY);
                }
                else if (strcmp(literal, VAR_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_VAR, line, current, (char*)VAR_KEY);
                }
                else if (strcmp(literal, WHILE_KEY) == 0)
                {
                    tokn = token_simple(TOKEN_WHILE, line, current, (char*)WHILE_KEY);
                }
                else
                {
                    tokn = token_simple(TOKEN_IDENTIFIER, line, current, literal);
                }
            }
            else
            {
                if (verbose)
                {
                    toknzr_error(line, current, c);
                }
                else
                {
                    tokn = token_simple(TOKEN_ERROR, line, current, "Unexpected character.");
                }
            }
            break;
        }
        fr(literal);
        literal = NULL;
        current++;
        if (tokn != NULL)
        {
            list_push(toknz.values, tokn);
            tokn = NULL;
        }
    }
    toknz.lines = line;
    list_push(toknz.values, token_simple(TOKEN_ENDOFFILE, line, current, (char*)"EOF"));
    return toknz;
}

static void tokens_foreach_token(List* tokens, void* tokenObj)
{
    Token* tokn = (Token*)tokenObj;
    token_destroy(tokn);
}

void toknzr_destroy(Tokenization toknz)
{
    list_foreach(toknz.values, tokens_foreach_token);
    list_destroy(toknz.values);
}
